import scrapy
from lofter.items import LofterItem

class UnicornSpider(scrapy.Spider):
    name = 'unicorn'
    allowed_domains = ['unicornwj.lofter.com']
    # start_urls = ['http://unicornwj.lofter.com/']

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
    }

    def start_requests(self):
        url = 'https://unicornwj.lofter.com/'
        yield scrapy.Request(url, callback=self.parse, headers=self.headers)

    def parse(self, response):
        # 在遇到不能整页面爬取的时候，就必须考虑一个一个爬取
        links = response.xpath('//a[@class="img"]/@href').getall()
        next_page = response.xpath('//a[@class="next"]/@href').get()
        # link = 'https://unicornwj.lofter.com/post/33a459_1ca79bde5'
        for link in links:
            yield response.follow(link, callback=self.parse2, headers=self.headers)

        next_page = 'https://unicornwj.lofter.com/' + next_page
        # next_page = 'https://unicornwj.lofter.com/?page=2&t=1586930400000'
        if next_page is not None:
            yield response.follow(next_page, callback=self.parse, headers=self.headers)

    def parse2(self, response):
        item = LofterItem()
        item['title'] = response.xpath('//div[@class="text"]/p[1]/text()').get()
        item['photo_link'] = response.xpath('//div[@class="pic"]/a/img/@src').getall()
        yield item

